import httplib
import os
from pyurl.url import BaseRemoteURL, scheme_registry
from pyurl.connection import Connection

def extract_links(file):
    list = []
    idx = 0
    while idx >= 0:
        href_idx = file.find("href=\"", idx)
        src_idx = file.find("src=\"", idx)
        if src_idx >= 0 and href_idx > src_idx:
            start = src_idx+len("src=\"")
        elif href_idx >= 0:
            start = href_idx+len("href=\"")
        else:
            break
        end = file.find("\"", start)
        list.append(file[start:end])
        idx = end
    return list

class HTTPConnection(Connection):
    def __init__(self):
        self.connection = None
        self.url = None
        
    def open(self, url):
        self.url = url
        self.connection = httplib.HTTPConnection(url.hostname, url.port or 80)
        self.connection.connect()
        
    def is_open(self):
        return self.connection is not None
    
    def close(self):
        if self.connection:
            self.connection.close()
            self.connection = None

    def exists(self, path):
        self.connection.request("HEAD", path)
        response = self.connection.getresponse()
        response.read()
        return response.status == 200
        
    def open_file(self, path, mode):
        self.connection.request("GET", path)
        response = self.connection.getresponse()
        if response.status == 200:
            file = os.tmpfile()
            file.write(response.read())
            file.seek(0)
            return file
        elif response.status == 301:
            response.read()
            return self.open_file(response.getheader("location"), mode)
        else: 
            response.read()
            raise IOError(2, "HTTP error "+str(response.status)+" "+response.reason+": '"+path+"'")


    def iter_folder(self, path):
        folder_url = str(self.url / path)
        file = self.open_file(path, "r").read()
        links = extract_links(file)
        
        def remove_get_parameter(url):
            idx = url.find("?")
            if idx >= 0:
                return url[0:idx]
            else:
                return url
        
        def check_url(url):
            if len(url) == 0:
                return False
            
            if url.startswith(path):
                return url.find("/", len(path)+1, len(url)-1) == -1
            elif url.startswith(folder_url):
                return url.find("/", len(folder_url)+1, len(url)-1) == -1
            elif not url.startswith("/"):
                return url.find("/") == -1
            
        def extract_names(url):
            name = url
            if url.startswith(path):
                name = url[len(path):]
            elif url.startswith(folder_url):
                name = url[len(folder_url):]
            
            if len(name) > 0 and name[len(name)-1] == "/":
                name = name[0:len(name)-1]
            return name
        
        def filter_invalid(url):
            if len(url) == 0:
                return False
            return True
        
        links = map(remove_get_parameter, links)
        links = filter(check_url, links)
        links = map(extract_names, links)
        links = filter(filter_invalid, links)
        links = set(links)
        return links
    
    
class HTTPURL(BaseRemoteURL):
    _scheme = "http"
    _connection_type = HTTPConnection
    
scheme_registry.register(HTTPURL)


if __name__ == "__main__":
    from pyurl import URL
    
    conn = HTTPConnection()
    conn.open(URL("http://www.dennis-kempin.de/various/"))
    print conn.iter_folder("/")
              